/*
 * Copyright 2011 Carnegie Mellon University
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *  
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, 
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package edu.cmu.lti.ritesdk;

import java.io.File;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;

import edu.cmu.lti.ritesdk.rite4qa.EvaluatorForRITE4QA;

/**
 * The Facade class with the main method for evaluation.
 * 
 * @author Hideki Shima
 *
 */
public class MainClassForEvaluation extends AbstractMainClass {

  @Override
  protected Options createOptions() {
    Options options = new Options();
    options.addOption("h", "help", false, "print this message and exit");
    options.addOption("t", "task", true, "task name (BC, MC, EXAM or RITE4QA)");
    options.addOption("r", "run-result", true, "run result file (or dir for batch evaluation)");
    options.addOption("i", "gold-standard", true, "gold standard file");
    return options;
  }

  @Override
  public void run(CommandLine cmd, Task task) throws Exception {
    String runPath  = cmd.getOptionValue("r");
    String goldPath = cmd.getOptionValue("i");
    File runTarget  = new File(runPath);
    File goldFile = new File(goldPath);
    
    File destDir;
    File[] runFiles;
    if (runTarget.isDirectory()) {
      System.out.println("Input dir: "+runTarget.getAbsolutePath());
      runFiles = runTarget.listFiles();
      destDir = new File("target/evaluation",runTarget.getName());
    } else {
      System.out.println("Input file: "+runTarget.getAbsolutePath());
      runFiles = new File[]{runTarget};
      destDir = new File("target/evaluation");
    }
    if (!destDir.exists()) {
      destDir.mkdirs();
    }

    DataSet goldData = FileUtils.readRiteXml(goldFile);
    
    StringBuilder sb = new StringBuilder();
    for ( File f : runFiles ) {
      try {
        AnalyzedDataSet analyzedData = FileUtils.readRunResult( f, goldData );
        // If the data is test data, we don't have gold labels so evaluation is skipped.
  //      boolean isTestData = Evaluator.haveLabels( analyzedData );
  //      if ( isTestData ) {
  //        System.out.println("ERROR: System labels not available in "+f.getName());
  //        continue;
  //      }
        {
          EvaluationResult evalResult = Evaluator.evaluate(analyzedData, task);
          
          sb.append("Accuracy = "+evalResult.getScore().get(Evaluator.ACCURACY)+" ("+f.getName()+")\n");
          File outputFile = new File(destDir,f.getName());
          FileUtils.writeStringToFile(evalResult.getEvaluationDetail(), outputFile);
        }
        if ( task.name().equalsIgnoreCase("RITE4QA") ) {
          EvaluationResult evalResult = EvaluatorForRITE4QA.evaluate(analyzedData, goldData);
          sb.append("MRR5     = "+evalResult.getScore().get(EvaluatorForRITE4QA.MRR5)+" ("+f.getName()+")\n");
          sb.append("TOP1     = "+evalResult.getScore().get(EvaluatorForRITE4QA.TOP1)+" ("+f.getName()+")\n");
          File outputFile = new File(destDir,"QAEval_"+f.getName());
          FileUtils.writeStringToFile(evalResult.getEvaluationDetail(), outputFile);
        }
      } catch ( Exception e ) {
        System.out.println("ERROR: occurred while processing: "+f);
        e.printStackTrace();
      }
    }
    System.out.print(sb);

    System.out.println("\n-------------------------------------------------------");
    System.out.println("Saved detailed evaluation result at:");
    System.out.println(destDir.getAbsolutePath()+"\n");
    System.out.println("Done running evaluation.");
    System.out.println("-------------------------------------------------------\n");
  }

  public static void main(String[] args) {
    new MainClassForEvaluation().run(args);
  }

}
